package com.xiaotu.spider.project.pageprocessor;

import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Component;

import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.FileUtil;
import com.xiaotu.common.util.GsonUtils;
import com.xiaotu.common.util.HtmlUtil;
import com.xiaotu.common.util.HttpUtils;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.spider.AUrlPageProcessor;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

/**
 * @类名 QQDocPageProcessor
 * @日期 2017年4月5日
 * @作者 高海军
 * @功能 腾讯视频纪录片爬取
 */
@Component("QQDocPageProcessor")
public class QQDocPageProcessor extends AUrlPageProcessor
{
	private static final String CSV_SPLITOR = "#***#";
	
	private static final String LABEL = "&label=";
	
	@Override
	public void process(Page page)
	{
		this.parsePage(page);
		this.addNextPage(page);
	}
	
	private void parsePage(Page page)
	{
		String url = page.getUrl().toString();
		String label = url.substring(url.length() - "人文".length());
		List<String> docList = page.getHtml()
				.xpath("//ul[@class='figures_list']/li[@class='list_item']")
				.all();
		docList.forEach(doc ->
		{
			Html docHtml = new Html(doc);
			String title = docHtml
					.xpath("//strong[@class='figure_title']/a/@title").get();
			String year = this.getYear(
					docHtml.xpath("//a[@class='figure']/@data-float").get());
			String play = docHtml.xpath("//span[@class='num']/text()").get();
			String score = HtmlUtil
					.delHTMLTag(
							docHtml.xpath("//div[@class='figure_score']").get())
					.replaceAll(" ", StringUtils.EMPTY);
			String desc = docHtml.xpath("//div[@class='figure_desc']/text()")
					.get();
			desc = desc == null ? StringUtils.EMPTY : desc;
			String line = label + CSV_SPLITOR + title + CSV_SPLITOR + year
					+ CSV_SPLITOR + play + CSV_SPLITOR + score + CSV_SPLITOR
					+ desc;
			System.out.println(line);
			FileUtil.writeFile("D://qq.csv", line + SepratorUtil.LINE_SEPRATOR);
		});
	}
	
	@SuppressWarnings("unchecked")
	private String getYear(String docId)
	{
		String data = HttpUtils.httpGetJSONP(
				"http://node.video.qq.com/x/api/float_vinfo2?callback=jQuery19109716476332649246_1500600956051&cid="
						+ docId + "&_=1500600956054");
		if (StringUtils.isEmpty(data))
			return StringUtils.EMPTY;
		Map<String, Object> map = GsonUtils.fromJson(
				data.substring(data.indexOf("(") + 1, data.lastIndexOf(")")),
				Map.class);
		Map<String, Object> cMap = (Map<String, Object>) map.get("c");
		if (cMap == null)
			return StringUtils.EMPTY;
		
		return cMap.get("year") == null ? StringUtils.EMPTY
				: cMap.get("year").toString();
	}
	
	private void addNextPage(Page page)
	{
		String href = page.getHtml()
				.xpath("//div[@class='mod_pages']/a[@class='page_next']/@href")
				.get();
		String label = page.getUrl().toString()
				.substring(page.getUrl().toString().indexOf(LABEL));
		
		if (StringUtils.isNotEmpty(href))
		{
			if (href.indexOf(LABEL) < 0)
				href += label;
			PageUtils.addTargetRequest(page, href,
					page.getRequest().getExtras());
		}
	}
	
	@Override
	protected String[] getStartRequestUrls()
	{
		return new String[] {
				"http://v.qq.com/x/list/doco?sort=18&itype=4&offset=0&label=自然"// ,
				// "http://v.qq.com/x/list/doco?sort=18&itype=10&offset=0&label=旅游",
				// "http://v.qq.com/x/list/doco?sort=18&offset=0&itype=5&label=人文"
		};
	}
	
	@Override
	protected String getSpiderType()
	{
		return "QQDoc";
	}
	
	@Override
	public String getPageDataType(Page page)
	{
		return Constants.PageDataType.SEARCH_LIST;
	}
	
}
